from typing import Dict, Any
import structlog
from scrapy.exceptions import DropItem


class ValidationPipeline:
    
    def __init__(self):
        self.logger = structlog.get_logger(self.__class__.__name__)
        self.stats = {
            "processed": 0,
            "dropped": 0,
            "validation_errors": {}
        }
    
    def process_item(self, item: Dict[str, Any], spider):
        self.stats["processed"] += 1
        
        # Required fields validation
        required_fields = ["url", "content", "source", "fetch_time"]
        missing_fields = []
        
        for field in required_fields:
            if field not in item or item[field] is None:
                missing_fields.append(field)
        
        if missing_fields:
            self.stats["dropped"] += 1
            error_key = "missing_fields"
            if error_key not in self.stats["validation_errors"]:
                self.stats["validation_errors"][error_key] = 0
            self.stats["validation_errors"][error_key] += 1
            
            self.logger.warning(
                "Dropping item due to missing required fields",
                url=item.get("url", "unknown"),
                missing_fields=missing_fields,
                spider=spider.name
            )
            raise DropItem(f"Missing required fields: {missing_fields}")
        
        # Content validation
        if "content" in item:
            content = item["content"]
            
            # Check for empty content
            if isinstance(content, dict):
                if not content.get("text") and not content.get("markdown"):
                    self.stats["dropped"] += 1
                    error_key = "empty_content"
                    if error_key not in self.stats["validation_errors"]:
                        self.stats["validation_errors"][error_key] = 0
                    self.stats["validation_errors"][error_key] += 1
                    
                    self.logger.warning(
                        "Dropping item due to empty content",
                        url=item.get("url"),
                        spider=spider.name
                    )
                    raise DropItem("Empty content")
            elif isinstance(content, str):
                if len(content.strip()) < 10:
                    self.stats["dropped"] += 1
                    error_key = "content_too_short"
                    if error_key not in self.stats["validation_errors"]:
                        self.stats["validation_errors"][error_key] = 0
                    self.stats["validation_errors"][error_key] += 1
                    
                    self.logger.warning(
                        "Dropping item due to content too short",
                        url=item.get("url"),
                        content_length=len(content),
                        spider=spider.name
                    )
                    raise DropItem("Content too short")
        
        # URL validation
        if "url" in item:
            url = item["url"]
            if not url.startswith(("http://", "https://")):
                self.stats["dropped"] += 1
                error_key = "invalid_url"
                if error_key not in self.stats["validation_errors"]:
                    self.stats["validation_errors"][error_key] = 0
                self.stats["validation_errors"][error_key] += 1
                
                self.logger.warning(
                    "Dropping item due to invalid URL",
                    url=url,
                    spider=spider.name
                )
                raise DropItem(f"Invalid URL: {url}")
        
        # Add validation status
        item["validation_status"] = "passed"
        
        self.logger.debug(
            "Item passed validation",
            url=item.get("url"),
            spider=spider.name
        )
        
        return item
    
    def close_spider(self, spider):
        self.logger.info(
            "Validation pipeline statistics",
            spider=spider.name,
            **self.stats
        )